import os,sys,re
import getopt
import csv, locale
from BeautifulSoup import BeautifulSoup as soup

class UnicodeWriter:
    #A CSV writer which will write rows to CSV file "f",
    #which is encoded in the given encoding.    
    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([s.encode("utf-8") for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)
			
class util_:
	def __init__(self) :
		self.files = []
		self.contents = []
		self.directories = []
		self.fileExtList = ['.jsp', '.html']
	def scan_file_folder(self, s_listdir='.'):
		self.contents= os.listdir( s_listdir)
		for i in self.contents:
			if os.path.isfile(i) == True :
				if os.path.splitext(i)[1] in self.fileExtList :
					self.files.append(i)
			elif os.path.isdir(i) == True :
				self.directories.append(i)
	def getfiles(self):
		return self.files
	def getfolders(self):
		return self.directories

class repertoire_:
	def __init__(self, id = 0):
		self.id=id
		self.path= ''
	def inc(self):
		return self.id + 1
	def getid(self):
		return self.id
		
class ligne_fichier_:
	def __init__(self, id = 0):
		self.id=id
		self.line_nbr = 0
	def inc(self):
		return self.id + 1
	def getline_nbr(self):
		return self.line_nbr
	def getid(self):
		return self.id
		
class script_:
	def __init__(self, id = 0):
		self.id=id
		self.Cdata = ''
		self.ligne_fichier= ligne_fichier_()
	def inc(self):
		return self.id + 1
	def getid(self):
		return self.id
	def getligne_fichier():
		return self.ligne_fichier.getline_nbr
		
class image_ :
	def __init__(self, id = 0):
		self.id=id
		self.width = 0
		self.height = 0
		self.src = ''
		self.ligne_fichier= ligne_fichier_()
	def inc(self):
		return self.id + 1
	def getid(self):
		return self.id
	def getligne_fichier():
		return self.ligne_fichier
		
class lien_ :
	def __init__(self, id = 0):
		self.id=id
		self.href = ''
		self.ligne_fichier= ligne_fichier_()		
	def inc(self):
		return self.id + 1
	def ishttp_link(self): 
		return (re.search('^http://',self.href)!= None)
	def isjavascript_link(self):
		return (re.search('^javascript:',self.href)!= None)
	def isanchor_link(self):
		return (re.match('^#[a-zA-Z0-9_]+',self.href) != None)
	
	def getid(self):
		return self.id		
	def getligne_fichier():
		return self.ligne_fichier
		
class fichier_temp_:
	def __init__(self, id = 0):
		#self.enc = ["ISO-8859-15"]#self.enc = ["windows-1252" ]
		self.id=id
		self.enc = ["UTF-8"]
		self.desc_commercial_csv_header=['//DESCRIPTIF_COMMERCIAL/TITRE','//DESCRIPTIF_COMMERCIAL/DEBUT_DE_VALIDITE','//DESCRIPTIF_COMMERCIAL/FIN_DE_VALIDITE','//DESCRIPTIF_COMMERCIAL/SECTION_IMPORTATION','//DESCRIPTIF_COMMERCIAL/SECTION_PRESENTE_OFFRE']
		self.nom = ''
		self.data= []
		self.result_iso =[]
		self.result_win =[]
		self.result_utf =[]
		self.liens = [lien_()]
		self.images = [image_()]
		self.scripts = [script_()]
	def inc(self):
		return self.id + 1
	def getid(self):
		return self.id
	def getliens(self):
		#print "la taille du tableau de liens %s" %len(self.liens)
		return self.liens
	def getdata(self):
		return self.data
	def getimages(self):
		return self.images
	def	cleanBeforeSetFilesResult(self):
		str =  ''.join(self.getdata())
		for encoded in self.enc : 
			html = soup(str, fromEncoding=encoded,  smartQuotesTo="html")
			html.__str__(encoded)
			for tag in html.findAll('td', {'class': True}):  
				if tag.attrMap['class'] == "docboxtitint": 
					tag.extract()
				else :
					pass
			if encoded == "ISO-8859-15":
				self.result_iso = html.__str__(encoded)
			if encoded == "windows-1252" : 
				self.result_win = html.__str__(encoded)
			if encoded == "UTF-8" : 
				self.result_utf = html.__str__(encoded)
	def setFilesResult(self):
		filename = self.nom.split('.',1)
		for encoded in self.enc :
			fileHandle = open ( filename[0] + '_htmlfrag_migrated_'+'id_'+ str(self.id)+ '_'+encoded +'.' + filename[1] , 'w' )
			if encoded == "ISO-8859-15":
				fileHandle.write( str(self.result_iso) )
			if encoded == "windows-1252" : 
				fileHandle.write( str(self.result_win) )
			if encoded == "UTF-8" : 
				fileHandle.write( str(self.result_utf) )
			fileHandle.close() 
	def setFilesImportCsv(self):
		filename = self.nom.split('.',1)
		for cvsheader in self.desc_commercial_csv_header :
			fileHandle = open ( filename[0] + '_csv_migrated_'+'id_'+self.id+ '_'+encoded +'.' + filename[1] , 'w' )
			if encoded == "ISO-8859-15":
				fileHandle.write( str(self.result_iso) )
			if encoded == "windows-1252" : 
				fileHandle.write( str(self.result_win) )
			if encoded == "UTF-8" : 
				fileHandle.write( str(self.result_utf) )
			fileHandle.close() 
	def set_all(self):
		self.setdata()
		self.setid()
	def setdata(self):
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		found =i =0
		while line:
			i=i+1
			if (re.search('boxdocmain', line)):
				print 'debut trouve'
				found =1
			if (re.search('Fin de boxdomain', line)):
				print 'fin trouve'
				found = 2
				break
			if found==1 :
				self.data.append(line.strip(' \t\n\r'))
			line=fileobj.readline()
	def setliens(self):
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		i =0
		while line:
			i=i+1
			html = soup(line)
			for tag_1 in html.findAll('input', {'name': True}):
				if tag_1.attrMap['name'] == "docid" : 
					self.id = tag_1.attrMap['value']

			for tag in html.findAll('a', {'href': True}):
				if tag.attrMap['href'] == ""  : 
					pass
				else :
					lien = lien_()
					lien.href = tag.attrMap['href']
					lien.ligne_fichier = i
					self.liens.append(lien)
			line=fileobj.readline()
	def setimages(self):
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		i=0
		while line:
			i=i+1
			html = soup(line)
			for tag in html.findAll('img', {'src': True}):
				if tag.attrMap['src'] == "" : 
					pass
				else :
					image = image_()
					image.src = tag.attrMap['src']
					image.ligne_fichier = i
					self.images.append(image)
			line=fileobj.readline()
	def setid(self) :
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		found =i =0
		while line:
			html = soup(line)
			for tag in html.findAll('input', {'name': True}):
				if tag.attrMap['name'] == "docid" : 
					self.id = tag.attrMap['value']
			line=fileobj.readline()
		
class fichier_orig_:
	def __init__(self, id = 0):
		self.id=id
		self.nom = ''
		self.liens = [lien_()]
		self.images = [image_()]
		self.scripts = [script_()]
	def inc(self):
		return self.id + 1
	def getid(self):
		return self.id
	def getliens(self):
		return self.liens
	def getimages(self):
		return self.images
	def setliens(self):
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		i =0
		while line:
			i=i+1
			html = soup(line)
			for tag in html.findAll('a', {'href': True}):
				if tag.attrMap['href'] == ""  : 
					pass
				else :
					lien = lien_()
					lien.href = tag.attrMap['href']
					lien.ligne_fichier = i
					self.liens.append(lien)
			line=fileobj.readline()
	def setimages(self):
		fileobj = open(self.nom,'r')
		line = fileobj.readline()
		i=0
		while line:
			i=i+1
			html = soup(line)
			for tag in html.findAll('img', {'src': True}):
				if tag.attrMap['src'] == "" : 
					pass
				else :
					image = image_()
					image.src = tag.attrMap['src']
					image.ligne_fichier = i
					self.images.append(image)
			line=fileobj.readline()
	
class migration_:
	def __init__(self, id = 0):
		self.id = id
		self.fichiers_temp = [fichier_temp_() ]
		self.fichiers_orig = [fichier_orig_()]
		self.repertoires = [repertoire_()]
	def inc(self):
		return self.id + 1
	def getStats(self):	
		for file in self.fichiers_orig:
			i=j=k=0
			for link in file.getliens():
				if link.isanchor_link():
					i=i+1
				if link.ishttp_link():
					j=j+1
				if link.isjavascript_link():
					k=k+1
			print "Pour %s il y a %s anchors, %s http et %s js dans les liens" %(file.nom,i,j,k)
	def getfiles_array_orig(self):
		return self.fichiers_orig
	def getfiles_array_temp(self):
		return self.fichiers_temp
	def setfiles_array_orig(self, rep):
		u = util_()
		u.scan_file_folder(rep)
		res = []
		for tuple in u.getfiles() : 
			f = fichier_orig_()
			f.nom = tuple
			f.setliens()
			f.setimages()
			res.append(f)
		self.fichiers_orig = res
	def setfiles_array_temp(self, rep):
		u = util_()
		u.scan_file_folder(rep)
		res = []
		for tuple in u.getfiles() : 
			r = fichier_temp_()
			r.nom = tuple
			r.set_all()
			#f.setimages()
			res.append(r)
		self.fichiers_temp = res

print locale.getdefaultlocale()

myMigration = migration_(0)
myMigration.setfiles_array_temp('.')
for fic in myMigration.getfiles_array_temp() : 
	fic.cleanBeforeSetFilesResult()
	fic.setFilesResult()
	#fic.setid()
	
#print myMigration.getfiles_array_temp()[].nom
#myMigration.getfiles_array_temp()[1].setid()

#myMigration.getfiles_array_temp()[1].cleanBeforeSetFilesResult()
#myMigration.getfiles_array_temp()[1].setFilesResult()
